# 数据可视化 实战练习
# 测试请求是否正常
import requests
# result = requests.get('https://baidu.com');
# print(result) 200 = 正常

# 数据处理
from bs4 import BeautifulSoup

# 模拟浏览器
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}

# 爬取请求路径
# 嗅事百科 - 仅用于学习

base_url = 'https://www.qiushibaike.com/text/page/'
for i in range(2, 10):
    context = '爬虫练习 嗅事百科 第{}页'.format(i)
    # 分页查询 循环页数
    result = requests.get(base_url + str(i), headers=headers)
    content = result.text
    # print(content)
    # 保存到BeautifulSoup
    soup = BeautifulSoup(content, 'lxml')
    # 分析返回html结构
    # 获取关键div class = article block untagged mb15 typs_hot 下面span标签
    # span内 的笑话就是我们想要的内容
    divs = soup.find_all(class_='article block untagged mb15 typs_hot')
    # print(divs)
    # 将文件写到桌面
    for i in divs:
        context = i.span.get_text()
        desktop_path = "C:\\Users\\liugui\\Desktop\\"  # 存放路径
        full_path = desktop_path + '嗅事百科.txt'
        with open(full_path, 'a', encoding='utf-8') as a:
            a.write(context)
